#  微博的爬取风景的案例
import requests
from lxml import etree
import json
import csv


f= open('微博大理洱海景区游客评论.csv', mode='a', encoding='utf-8', newline='')
csv_writer = csv.writer(f)
csv_writer.writerow( ['用户名', '性别',  '所属地区','设备类型', '用户发表内容'])
def listInfo(url):
    headers={
        'User-agent': '',
        'accept': '',
        'cookie': ''
             }
    response = requests.get(url=url,headers=headers)
    html0 = etree.HTML(response.text)
    page = html0.xpath('//div[@class="card-wrap"]/div[@class="card"]')

    for zzz in page:
        html1 = etree.HTML(etree.tounicode(zzz))
        # sss=html1.xpath('//div[@class="card-feed"]/div[@class="content"]/div[@class="info"]/div[2]/a[1]')
        userUrl="https:"+html1.xpath('//div[@class="card-feed"]/div[@class="content"]/div[@class="info"]/div[2]/a[1]/@href')[0]
        userName=html1.xpath('//div[@class="card-feed"]/div[@class="content"]/div[@class="info"]/div[2]/a[1]/text()')[0]
        usertext=html1.xpath('//div[@class="card-feed"]/div[@class="content"]/p[@class="txt"]/text()')
        try: device = html1.xpath('//div[@class="card-feed"]/div[@class="content"]/p[@class="from"]/a/text()')[1]
        except IndexError: device = None
        userId=userUrl.replace('https://weibo.com/','')
        userId=userId.split("?")[0]
        user= getUser(userId)
        user['device'] = device
        user['usertext']=usertext
        csv_writer.writerow([ user['screen_name'], user['gender'], user['location'], user['device'], user['usertext']])
        print(user)
    nextUrl=html0.xpath('//a[@class="next"]/@href')
    if len(nextUrl) > 0:
       return 'https://s.weibo.com/'+nextUrl[0]
    return "";
def getUser(userId):
    url= "https://weibo.com/ajax/profile/info?custom="+userId
    headers={
         'User-agent': '',
        'accept': '',
        'cookie': ''
        }
    response = requests.get(url=url, headers=headers)
    userInfo=json.loads(response.text);
    user={}
    user['screen_name']=(userInfo['data']['user']['screen_name'])
    user['location']=(userInfo['data']['user']['location'])
    user['gender']=(userInfo['data']['user']['gender'])
    return user
def main():
    nexturl= ''
    while(nexturl):
        print(nexturl)
        nexturl=listInfo(nexturl)
main()